{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to DataFrames\n",
    "**[Bogumił Kamiński](http://bogumilkaminski.pl/about/), Apr 21, 2018**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DataFrames # load package"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Working with CategoricalArrays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Constructor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element CategoricalArrays.CategoricalArray{String,1,UInt32}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"C\""
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = categorical([\"A\", \"B\", \"B\", \"C\"]) # unordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element CategoricalArrays.CategoricalArray{String,1,UInt32}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"C\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = categorical([\"A\", \"B\", \"B\", \"C\"], ordered=true) # ordered, by default order is sorting order"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element CategoricalArrays.CategoricalArray{Union{Missings.Missing, String},1,UInt32}:\n",
       " \"A\"    \n",
       " \"B\"    \n",
       " \"B\"    \n",
       " \"C\"    \n",
       " missing"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z = categorical([\"A\",\"B\",\"B\",\"C\", missing]) # unordered with missings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10-element CategoricalArrays.CategoricalArray{String,1,UInt32}:\n",
       " \"[1.0, 2.8)\" \n",
       " \"[1.0, 2.8)\" \n",
       " \"[2.8, 4.6)\" \n",
       " \"[2.8, 4.6)\" \n",
       " \"[4.6, 6.4)\" \n",
       " \"[4.6, 6.4)\" \n",
       " \"[6.4, 8.2)\" \n",
       " \"[6.4, 8.2)\" \n",
       " \"[8.2, 10.0]\"\n",
       " \"[8.2, 10.0]\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = cut(1:10, 5) # ordered, into equal counts, possible to rename labels and give custom breaks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>n</th></tr></thead><tbody><tr><th>1</th><td>[-4.25932, -1.2841)</td><td>10000</td></tr><tr><th>2</th><td>[-1.2841, -0.845596)</td><td>10000</td></tr><tr><th>3</th><td>[-0.845596, -0.525619)</td><td>10000</td></tr><tr><th>4</th><td>[-0.525619, -0.251247)</td><td>10000</td></tr><tr><th>5</th><td>[-0.251247, 0.00327741)</td><td>10000</td></tr><tr><th>6</th><td>[0.00327741, 0.256183)</td><td>10000</td></tr><tr><th>7</th><td>[0.256183, 0.530553)</td><td>10000</td></tr><tr><th>8</th><td>[0.530553, 0.848815)</td><td>10000</td></tr><tr><th>9</th><td>[0.848815, 1.28808)</td><td>10000</td></tr><tr><th>10</th><td>[1.28808, 4.27432]</td><td>10000</td></tr></tbody></table>"
      ],
      "text/plain": [
       "10×2 DataFrames.DataFrame\n",
       "│ Row │ x                       │ n     │\n",
       "├─────┼─────────────────────────┼───────┤\n",
       "│ 1   │ [-4.25932, -1.2841)     │ 10000 │\n",
       "│ 2   │ [-1.2841, -0.845596)    │ 10000 │\n",
       "│ 3   │ [-0.845596, -0.525619)  │ 10000 │\n",
       "│ 4   │ [-0.525619, -0.251247)  │ 10000 │\n",
       "│ 5   │ [-0.251247, 0.00327741) │ 10000 │\n",
       "│ 6   │ [0.00327741, 0.256183)  │ 10000 │\n",
       "│ 7   │ [0.256183, 0.530553)    │ 10000 │\n",
       "│ 8   │ [0.530553, 0.848815)    │ 10000 │\n",
       "│ 9   │ [0.848815, 1.28808)     │ 10000 │\n",
       "│ 10  │ [1.28808, 4.27432]      │ 10000 │"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by(DataFrame(x=cut(randn(100000), 10)), :x, d -> DataFrame(n=nrow(d)), sort=true) # just to make sure it works right"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element CategoricalArrays.CategoricalArray{Int64,1,UInt32}:\n",
       " 1\n",
       " 2\n",
       " 2\n",
       " 3\n",
       " 3"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v = categorical([1,2,2,3,3]) # contains integers not strings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Array{Union{Missings.Missing, String},1}:\n",
       " \"A\"    \n",
       " \"B\"    \n",
       " \"B\"    \n",
       " \"C\"    \n",
       " missing"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Vector{Union{String, Missing}}(z) # sometimes you need to convert back to a standard vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Managing levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Array{CategoricalArrays.CategoricalArray{T,1,UInt32,V,C,U} where U where C where V where T,1}:\n",
       " CategoricalArrays.CategoricalString{UInt32}[\"A\", \"B\", \"B\", \"C\"]                                                                                                                          \n",
       " CategoricalArrays.CategoricalString{UInt32}[\"A\", \"B\", \"B\", \"C\"]                                                                                                                          \n",
       " Union{CategoricalArrays.CategoricalString{UInt32}, Missings.Missing}[\"A\", \"B\", \"B\", \"C\", missing]                                                                                        \n",
       " CategoricalArrays.CategoricalString{UInt32}[\"[1.0, 2.8)\", \"[1.0, 2.8)\", \"[2.8, 4.6)\", \"[2.8, 4.6)\", \"[4.6, 6.4)\", \"[4.6, 6.4)\", \"[6.4, 8.2)\", \"[6.4, 8.2)\", \"[8.2, 10.0]\", \"[8.2, 10.0]\"]\n",
       " CategoricalArrays.CategoricalValue{Int64,UInt32}[1, 2, 2, 3, 3]                                                                                                                          "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr = [x,y,z,c,v]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element BitArray{1}:\n",
       " false\n",
       "  true\n",
       " false\n",
       "  true\n",
       " false"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isordered.(arr) # chcek if categorical array is orderd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalArrays.CategoricalString{UInt32}[\"A\", \"B\", \"B\", \"C\"], true)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered!(x, true), isordered(x) # make x ordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalArrays.CategoricalString{UInt32}[\"A\", \"B\", \"B\", \"C\"], false)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ordered!(x, false), isordered(x) # and unordered again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Array{Array{T,1} where T,1}:\n",
       " String[\"A\", \"B\", \"C\"]                                                        \n",
       " String[\"A\", \"B\", \"C\"]                                                        \n",
       " String[\"A\", \"B\", \"C\"]                                                        \n",
       " String[\"[1.0, 2.8)\", \"[2.8, 4.6)\", \"[4.6, 6.4)\", \"[6.4, 8.2)\", \"[8.2, 10.0]\"]\n",
       " [1, 2, 3]                                                                    "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels.(arr) # list levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Array{Array{T,1} where T,1}:\n",
       " String[\"A\", \"B\", \"C\"]                                                        \n",
       " String[\"A\", \"B\", \"C\"]                                                        \n",
       " Union{Missings.Missing, String}[\"A\", \"B\", \"C\", missing]                      \n",
       " String[\"[1.0, 2.8)\", \"[2.8, 4.6)\", \"[4.6, 6.4)\", \"[6.4, 8.2)\", \"[8.2, 10.0]\"]\n",
       " [1, 2, 3]                                                                    "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique.(arr) # missing will be included"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[1] < y[2] # can compare as y is ordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mArgumentError: Unordered CategoricalValue objects cannot be tested for order using <. Use isless instead, or call the ordered! function on the parent array to change this\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mArgumentError: Unordered CategoricalValue objects cannot be tested for order using <. Use isless instead, or call the ordered! function on the parent array to change this\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1m<\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::CategoricalArrays.CategoricalValue{Int64,UInt32}, ::CategoricalArrays.CategoricalValue{Int64,UInt32}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1mD:\\Software\\JULIA_PKG\\v0.6\\CategoricalArrays\\src\\value.jl:149\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "v[1] < v[2] # not comparable, v is unordered although it contains integers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element CategoricalArrays.CategoricalArray{String,1,UInt32}:\n",
       " \"A\"\n",
       " \"B\"\n",
       " \"B\"\n",
       " \"C\""
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels!(y, [\"C\", \"B\", \"A\"]) # you can reorder levels, mostly useful for ordered CategoricalArrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[1] < y[2] # observe that the order is changed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mArgumentError: cannot remove level \"C\" as it is used at position 4 and allow_missing=false.\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mArgumentError: cannot remove level \"C\" as it is used at position 4 and allow_missing=false.\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1m#levels!#55\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Bool, ::Function, ::CategoricalArrays.CategoricalArray{Union{Missings.Missing, String},1,UInt32,String,CategoricalArrays.CategoricalString{UInt32},Missings.Missing}, ::Array{String,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1mD:\\Software\\JULIA_PKG\\v0.6\\CategoricalArrays\\src\\array.jl:588\u001b[22m\u001b[22m",
      " [2] \u001b[1mlevels!\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::CategoricalArrays.CategoricalArray{Union{Missings.Missing, String},1,UInt32,String,CategoricalArrays.CategoricalString{UInt32},Missings.Missing}, ::Array{String,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1mD:\\Software\\JULIA_PKG\\v0.6\\CategoricalArrays\\src\\array.jl:576\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "levels!(z, [\"A\", \"B\"]) # you have to specify all levels that are present"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element CategoricalArrays.CategoricalArray{Union{Missings.Missing, String},1,UInt32}:\n",
       " \"A\"    \n",
       " \"B\"    \n",
       " \"B\"    \n",
       " missing\n",
       " missing"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels!(z, [\"A\", \"B\"], allow_missing=true) # unless the underlying array allows for missings and force removal of levels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element CategoricalArrays.CategoricalArray{Union{Missings.Missing, String},1,UInt32}:\n",
       " \"B\"    \n",
       " \"B\"    \n",
       " \"B\"    \n",
       " missing\n",
       " missing"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z[1] = \"B\"\n",
    "z # now z has only \"B\" entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2-element Array{String,1}:\n",
       " \"A\"\n",
       " \"B\""
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "levels(z) # but it remembers the levels it had (the reason is mostly performance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1-element Array{String,1}:\n",
       " \"B\""
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "droplevels!(z) # this way we can clean it up\n",
    "levels(z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data manipulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalArrays.CategoricalString{UInt32}[\"A\", \"B\", \"B\", \"C\"], String[\"A\", \"B\", \"C\"])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, levels(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalArrays.CategoricalString{UInt32}[\"A\", \"0\", \"B\", \"C\"], String[\"A\", \"B\", \"C\", \"0\"])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[2] = \"0\"\n",
    "x, levels(x) # new level added at the end (works only for unordered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(CategoricalArrays.CategoricalValue{Int64,UInt32}[1, 2, 2, 3, 3], [1, 2, 3])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v, levels(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mMethodError: no method matching +(::CategoricalArrays.CategoricalValue{Int64,UInt32}, ::CategoricalArrays.CategoricalValue{Int64,UInt32})\u001b[0m\nClosest candidates are:\n  +(::Any, ::Any, \u001b[91m::Any\u001b[39m, \u001b[91m::Any...\u001b[39m) at operators.jl:424\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mMethodError: no method matching +(::CategoricalArrays.CategoricalValue{Int64,UInt32}, ::CategoricalArrays.CategoricalValue{Int64,UInt32})\u001b[0m\nClosest candidates are:\n  +(::Any, ::Any, \u001b[91m::Any\u001b[39m, \u001b[91m::Any...\u001b[39m) at operators.jl:424\u001b[39m",
      ""
     ]
    }
   ],
   "source": [
    "v[1] + v[2] # even though underlying data is Int, we cannot operate on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Array{Int64,1}:\n",
       " 1\n",
       " 2\n",
       " 2\n",
       " 3\n",
       " 3"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Vector{Int}(v) # you have either to retrieve the data by conversion (may be expensive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get(v[1]) + get(v[2]) # or get a single value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Array{Int64,1}:\n",
       " 1\n",
       " 2\n",
       " 2\n",
       " 3\n",
       " 3"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get.(v) # this will work for arrays witout missings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mMethodError: no method matching get(::Missings.Missing)\u001b[0m\nClosest candidates are:\n  get(\u001b[91m::ObjectIdDict\u001b[39m, \u001b[91m::ANY\u001b[39m, \u001b[91m::ANY\u001b[39m) at associative.jl:434\n  get(\u001b[91m::Base.EnvHash\u001b[39m, \u001b[91m::AbstractString\u001b[39m, \u001b[91m::Any\u001b[39m) at env.jl:79\n  get(\u001b[91m::ZMQ.Context\u001b[39m, \u001b[91m::Integer\u001b[39m) at D:\\Software\\JULIA_PKG\\v0.6\\ZMQ\\src\\ZMQ.jl:136\n  ...\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mMethodError: no method matching get(::Missings.Missing)\u001b[0m\nClosest candidates are:\n  get(\u001b[91m::ObjectIdDict\u001b[39m, \u001b[91m::ANY\u001b[39m, \u001b[91m::ANY\u001b[39m) at associative.jl:434\n  get(\u001b[91m::Base.EnvHash\u001b[39m, \u001b[91m::AbstractString\u001b[39m, \u001b[91m::Any\u001b[39m) at env.jl:79\n  get(\u001b[91m::ZMQ.Context\u001b[39m, \u001b[91m::Integer\u001b[39m) at D:\\Software\\JULIA_PKG\\v0.6\\ZMQ\\src\\ZMQ.jl:136\n  ...\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1mmacro expansion\u001b[22m\u001b[22m at \u001b[1m.\\broadcast.jl:155\u001b[22m\u001b[22m [inlined]",
      " [2] \u001b[1mmacro expansion\u001b[22m\u001b[22m at \u001b[1m.\\simdloop.jl:73\u001b[22m\u001b[22m [inlined]",
      " [3] \u001b[1mmacro expansion\u001b[22m\u001b[22m at \u001b[1m.\\broadcast.jl:149\u001b[22m\u001b[22m [inlined]",
      " [4] \u001b[1m_broadcast!\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Base.#get, ::Array{String,1}, ::Tuple{Tuple{Bool}}, ::Tuple{Tuple{Int64}}, ::CategoricalArrays.CategoricalArray{Union{Missings.Missing, String},1,UInt32,String,CategoricalArrays.CategoricalString{UInt32},Missings.Missing}, ::Tuple{}, ::Type{Val{0}}, ::CartesianRange{CartesianIndex{1}}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m.\\broadcast.jl:141\u001b[22m\u001b[22m",
      " [5] \u001b[1mbroadcast_t\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Function, ::Type{T} where T, ::Tuple{Base.OneTo{Int64}}, ::CartesianRange{CartesianIndex{1}}, ::CategoricalArrays.CategoricalArray{Union{Missings.Missing, String},1,UInt32,String,CategoricalArrays.CategoricalString{UInt32},Missings.Missing}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m.\\broadcast.jl:270\u001b[22m\u001b[22m",
      " [6] \u001b[1mbroadcast_c\u001b[22m\u001b[22m at \u001b[1m.\\broadcast.jl:316\u001b[22m\u001b[22m [inlined]",
      " [7] \u001b[1mbroadcast\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Function, ::CategoricalArrays.CategoricalArray{Union{Missings.Missing, String},1,UInt32,String,CategoricalArrays.CategoricalString{UInt32},Missings.Missing}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m.\\broadcast.jl:455\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "get.(z) # but will fail on missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5-element Array{Union{Missings.Missing, String},1}:\n",
       " \"B\"    \n",
       " \"B\"    \n",
       " \"B\"    \n",
       " missing\n",
       " missing"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Vector{Union{String, Missing}}(z) # you have to do the conversion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\"BB\", Any[\"BB\", \"BB\", \"BB\", missing, missing])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z[1]*z[2], z.^2 # the only exception are CategoricalArrays based on String - you can operate on them normally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Array{Union{Int64, Missings.Missing},1}:\n",
       " 10       \n",
       "  2       \n",
       "  3       \n",
       "  4       \n",
       "  5       \n",
       "   missing"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recode([1,2,3,4,5,missing], 1=>10) # recode some values in an array; has also in place recode! equivalent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Array{Any,1}:\n",
       " 10       \n",
       " 20       \n",
       "   \"a\"    \n",
       "   \"a\"    \n",
       "   \"a\"    \n",
       "   missing"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recode([1,2,3,4,5,missing], \"a\", 1=>10, 2=>20) # here we provided a default value for not mapped recodings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6-element Array{Any,1}:\n",
       " 10         \n",
       "  2         \n",
       "  3         \n",
       "  4         \n",
       "  5         \n",
       "   \"missing\""
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recode([1,2,3,4,5,missing], 1=>10, missing=>\"missing\") # to recode Missing you have to do it explicitly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Union{CategoricalArrays.CategoricalValue{Int64,UInt32}, Missings.Missing}[1, 2, 3, 4, 5, missing], [1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = categorical([1:5; missing])\n",
    "t, levels(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Union{CategoricalArrays.CategoricalValue{Int64,UInt32}, Missings.Missing}[2, 2, 2, 4, 5, missing], [2, 4, 5])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recode!(t, [1,3]=>2)\n",
    "t, levels(t) # note that the levels are dropped after recode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3-element Array{Int64,1}:\n",
       "  3\n",
       "  0\n",
       " -1"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = categorical([1,2,3], ordered=true)\n",
    "levels(recode(t, 2=>0, 1=>-1)) # and if you introduce a new levels they are added at the end in the order of appearance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3-element Array{Int64,1}:\n",
       " 100\n",
       " 200\n",
       " 300"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = categorical([1,2,3,4,5], ordered=true) # when using default it becomes the last level\n",
    "levels(recode(t, 300, [1,2]=>100, 3=>200))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparisons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4×4 Array{Bool,2}:\n",
       " true  true  true  true\n",
       " true  true  true  true\n",
       " true  true  true  true\n",
       " true  true  true  true"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = categorical([1,2,3])\n",
    "xs = [x, categorical(x), categorical(x, ordered=true), categorical(x, ordered=true)]\n",
    "levels!(xs[2], [3,2,1])\n",
    "levels!(xs[4], [2,3,1])\n",
    "[a == b for a in xs, b in xs] # all are equal - comparison only by contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4×4 Array{Bool,2}:\n",
       "  true  false  false  false\n",
       " false   true  false  false\n",
       " false  false   true  false\n",
       " false  false  false   true"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "signature(x::CategoricalArray) = (x, levels(x), isordered(x)) # this is actually the full signature of CategoricalArray\n",
    "# all are different, notice that x[1] and x[2] are unordered but have a different order of levels\n",
    "[signature(a) == signature(b) for a in xs, b in xs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mArgumentError: Unordered CategoricalValue objects cannot be tested for order using <. Use isless instead, or call the ordered! function on the parent array to change this\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mArgumentError: Unordered CategoricalValue objects cannot be tested for order using <. Use isless instead, or call the ordered! function on the parent array to change this\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1m<\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::CategoricalArrays.CategoricalValue{Int64,UInt32}, ::CategoricalArrays.CategoricalValue{Int64,UInt32}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1mD:\\Software\\JULIA_PKG\\v0.6\\CategoricalArrays\\src\\value.jl:149\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "x[1] < x[2] # you cannot compare elements of unordered CategoricalArray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t[1] < t[2] # but you can do it for an ordered one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isless(x[1], x[2]) # isless works within the same CategoricalArray even if it is not ordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mArgumentError: CategoricalValue objects with different pools cannot be tested for order\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mArgumentError: CategoricalValue objects with different pools cannot be tested for order\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1misless\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::CategoricalArrays.CategoricalValue{Int64,UInt32}, ::CategoricalArrays.CategoricalValue{Int64,UInt32}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1mD:\\Software\\JULIA_PKG\\v0.6\\CategoricalArrays\\src\\value.jl:139\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "y = deepcopy(x) # but not across categorical arrays\n",
    "isless(x[1], y[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isless(get(x[1]), get(y[2])) # you can use get to make a comparison of the contents of CategoricalArray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "false"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[1] == y[2] # equality tests works OK across CategoricalArrays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical columns in a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th><th>z</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>'a'</td><td>a</td></tr><tr><th>2</th><td>2</td><td>'b'</td><td>b</td></tr><tr><th>3</th><td>3</td><td>'c'</td><td>c</td></tr></tbody></table>"
      ],
      "text/plain": [
       "3×3 DataFrames.DataFrame\n",
       "│ Row │ x │ y   │ z │\n",
       "├─────┼───┼─────┼───┤\n",
       "│ 1   │ 1 │ 'a' │ a │\n",
       "│ 2   │ 2 │ 'b' │ b │\n",
       "│ 3   │ 3 │ 'c' │ c │"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = DataFrame(x = 1:3, y = 'a':'c', z = [\"a\",\"b\",\"c\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th><th>z</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>'a'</td><td>a</td></tr><tr><th>2</th><td>2</td><td>'b'</td><td>b</td></tr><tr><th>3</th><td>3</td><td>'c'</td><td>c</td></tr></tbody></table>"
      ],
      "text/plain": [
       "3×3 DataFrames.DataFrame\n",
       "│ Row │ x │ y   │ z │\n",
       "├─────┼───┼─────┼───┤\n",
       "│ 1   │ 1 │ 'a' │ a │\n",
       "│ 2   │ 2 │ 'b' │ b │\n",
       "│ 3   │ 3 │ 'c' │ c │"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical!(df) # converts all eltype(AbstractString) columns to categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3×3 DataFrames.DataFrame\n",
      "│ Col # │ Name │ Eltype                                      │ Missing │\n",
      "├───────┼──────┼─────────────────────────────────────────────┼─────────┤\n",
      "│ 1     │ x    │ Int64                                       │ 0       │\n",
      "│ 2     │ y    │ Char                                        │ 0       │\n",
      "│ 3     │ z    │ CategoricalArrays.CategoricalString{UInt32} │ 0       │\n",
      "\n",
      "│ Col # │ Values      │\n",
      "├───────┼─────────────┤\n",
      "│ 1     │ 1  …  3     │\n",
      "│ 2     │ 'a'  …  'c' │\n",
      "│ 3     │ a  …  c     │"
     ]
    }
   ],
   "source": [
    "showcols(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>x</th><th>y</th><th>z</th></tr></thead><tbody><tr><th>1</th><td>1</td><td>'a'</td><td>a</td></tr><tr><th>2</th><td>2</td><td>'b'</td><td>b</td></tr><tr><th>3</th><td>3</td><td>'c'</td><td>c</td></tr></tbody></table>"
      ],
      "text/plain": [
       "3×3 DataFrames.DataFrame\n",
       "│ Row │ x │ y   │ z │\n",
       "├─────┼───┼─────┼───┤\n",
       "│ 1   │ 1 │ 'a' │ a │\n",
       "│ 2   │ 2 │ 'b' │ b │\n",
       "│ 3   │ 3 │ 'c' │ c │"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical!(df, :x) # manually convert to categorical column :x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3×3 DataFrames.DataFrame\n",
      "│ Col # │ Name │ Eltype                                           │ Missing │\n",
      "├───────┼──────┼──────────────────────────────────────────────────┼─────────┤\n",
      "│ 1     │ x    │ CategoricalArrays.CategoricalValue{Int64,UInt32} │ 0       │\n",
      "│ 2     │ y    │ Char                                             │ 0       │\n",
      "│ 3     │ z    │ CategoricalArrays.CategoricalString{UInt32}      │ 0       │\n",
      "\n",
      "│ Col # │ Values      │\n",
      "├───────┼─────────────┤\n",
      "│ 1     │ 1  …  3     │\n",
      "│ 2     │ 'a'  …  'c' │\n",
      "│ 3     │ a  …  c     │"
     ]
    }
   ],
   "source": [
    "showcols(df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.2",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
